
from IPython.core.display import HTML
display(HTML("""
<script>
code_show=true;
function code_toggle() {
 if (code_show){
 $('div.input').show();
 } else {
 $('div.input').hide();
 }
 code_show = !code_show
}
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()">
<input type="submit" value="Click here to toggle on/off the raw code.">
</form>"""))


from textwrap import wrap
import requests
import sqlite3
from sqlalchemy import create_engine
import bs4
import json
import re
import time
import pickle
from PIL import Image
from tqdm import tqdm
from tqdm.notebook import trange

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import NMF

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize, RegexpTokenizer

from nltk.stem.snowball import SnowballStemmer
from nltk.stem import PorterStemmer, WordNetLemmatizer

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator


def clean_text(text, stop_words, lemmatizer):
    """
    Preprocess the text using lemmatization.
    """
    text = text.casefold()
    text_list = [
        lemmatizer.lemmatize(word)
        for word in re.findall(r'\b[a-z-]+\b', text)
        if word not in stop_words
    ]
    return ' '.join(text_list)

def vectorize(corpus, model, params):
    """
    Vectorize the corpus of articles using `model`.
    """
    vectorizer = model(**params)
    sparse_corpus = vectorizer.fit_transform(corpus)
    corpus_df = pd.DataFrame.sparse.from_spmatrix(
        sparse_corpus, columns=vectorizer.get_feature_names_out()
    )
    return corpus_df

def get_token_counts(df):
    """
    Return a series of the sum of weights of each token in the TF-IDF
    vectorized corpus.
    """
    return df.sum().sort_values(ascending=False)

def plot_top_words(model, feature_names, n_lf, n_top_words):
    """
    Plot the `n_top_words` in the NMF `model` for `n_lf` latent factors.
    """
    rows = int(np.ceil(n_lf/6))
    extra_axes = rows*6-n_lf
    fig, axes = plt.subplots(rows, 6, figsize=(30, 8*rows), sharex=True)
    axes = axes.flatten()
    for i in range(1, extra_axes+1):
        fig.delaxes(axes[-i])
    for topic_idx, topic in enumerate(model.components_):
        top_features_ind = topic.argsort()[:-n_top_words-1:-1]
        top_features = [feature_names[i] for i in top_features_ind]
        weights = topic[top_features_ind]

        ax = axes[topic_idx]
        ax.barh(top_features, weights, height=0.7)
        ax.set_title(f'LF{topic_idx+1}', fontdict={"fontsize": 30})
        ax.invert_yaxis()
        ax.tick_params(axis="both", which="major", labelsize=20)
        for i in "top right left".split():
            ax.spines[i].set_visible(False)
#     fig.suptitle(title, fontsize=40)

    plt.subplots_adjust(top=0.8, bottom=0.05, wspace=0.90, hspace=0.3)
    plt.show()
    
def plot_top_words_wc(model, feature_names, n_lf, n_top_words, lf_names,
                      mask):
    """
    Plot the `n_top_words` in the NMF `model` for `n_lf` latent factors.
    """
    def black_color_func(word, font_size, position, orientation,
                         random_state=None, **kwargs):
        return ("hsl(0,100%, 1%)")
    rows = int(np.ceil(n_lf/3))
    extra_axes = rows*3-n_lf
    fig, axes = plt.subplots(rows, 3, figsize=(15, 5*rows), sharex=True)
    axes = axes.flatten()
    for i in range(1, extra_axes+1):
        fig.delaxes(axes[-i])
    for topic_idx, topic in enumerate(model.components_):
        top_features_ind = topic.argsort()[:-n_top_words-1:-1]
        top_features = [feature_names[i] for i in top_features_ind]
        weights = topic[top_features_ind]
        
        wc = WordCloud(
            mask=mask,
            background_color='white',
            max_words=300,
            collocations = False,
            stopwords=stop_words,
            random_state=topic_idx+143,
        )
        
        tokens = pd.Series(weights, index=top_features)
        ax = axes[topic_idx]
        ax.imshow(wc.generate_from_frequencies(tokens),
                  interpolation='spline16')
        ax.axis('off')
        ax.set_title(lf_names[topic_idx], fontdict={"fontsize": 30})
        ax.imshow(wc.recolor(color_func=black_color_func))
#     fig.suptitle(title, fontsize=40)
    fig.tight_layout(pad=2)
#     plt.subplots_adjust(top=0.8, bottom=0.05, wspace=0.90, hspace=0.3)
    plt.show()

def plot_dist_year(year_dict, lf_names):
    """
    Plot the `n_top_words` in the NMF `model` for `n_lf` latent factors.
    """
    fig, axes = plt.subplots(2, 3, figsize=(22, 18))
#     fig.suptitle(title, fontsize=40)
    fig.tight_layout(pad=8)
    axes = axes.flatten()
    for yr, ax in zip(year_dict.keys(), axes):
        counts = (pd.Series(year_dict[yr]['W'][0].argmax(axis=1))
                  .replace(lf_names[yr])
                  .value_counts()
                  .sort_values()
                  .reset_index()
                  .to_numpy())
        ax.barh(
            [comp for comp in counts[:, 0]],
            counts[:, 1]
        )
        ax.set_title(f'{yr}', fontsize=20)
#         ax.set_ylabel('Topic', fontsize=15)
        ax.set_xlabel('Number of Articles', fontsize=20)
        ax.tick_params(axis="both", which="major", labelsize=15)
    fig.subplots_adjust(wspace=0.5)
    plt.show()

fig_num = 2
def fig_label(title, caption):
    global fig_num
    """
    Print figure caption on jupyter notebook
    """
    display(HTML(f"""<p style="font-size:14px;font-style:default;"><b>
                     Figure {fig_num}. {title}.</b><br>{caption}</p>"""))
    fig_num += 1
    
table_num = 1
def table_label(title, caption):
    global table_num
    """
    Print table caption on jupyter notebook
    """
    display(HTML(f"""<center style="font-size:14px;font-style:default;"><b>
                     Table {table_num}. {title}.</b><br>{caption}</center>""")
           )
    table_num += 1


# Load the dataset.
metro = pd.read_csv('all_scraped_metro_news.csv')

# Filter by date (include only Duterte admin time).
metro.date = pd.to_datetime(metro.date)
metro = metro[(metro.date < '2022-06-30') & (metro.date >= '2016-06-30')]

# Remove 'did you know' articles because they are not news.
metro = (metro[~(metro.title.str.lower().str.contains('did you know'))]
         .sort_values('date')
         .reset_index(drop=True))

# Remove the promotion Subscribe to INQUIRER PLUS at the end of the texts.
metro.text = metro.text.apply(lambda x: x[:x.find('Subscribe')])

# Regroup the articles (new year starts after June 30).
indices = [0]
years = [2016, 2017, 2018, 2019, 2020, 2021]
for year in years:
    indices.append(
        metro[(metro.date < f'{year+1}-06-30')].iloc[-1].name
        + 1 - sum(indices)
    )
indices = indices[1:]

# Overwrite the year feature with the new groupings.
year_list = []
for idx, year in zip(indices, years):
    year_list.extend([year]*idx)
    
metro.year = pd.Series(year_list)

# Remove article with no text.
metro = metro.drop(index=4059).reset_index()

metro = metro[['title', 'date', 'year', 'link', 'text']]


text_lengths = metro.text.apply(lambda x: len(x.split()))

fig, ax = plt.subplots(figsize=(10, 6))
text_lengths.hist(bins=30, ax=ax)
ax.set_xlabel('Word Count', fontsize=15)
ax.set_ylabel('Number of Articles', fontsize=15)
plt.show()

fig_label('Distribution of Articles in the Corpus by Word Count',
          'There are 6960 articles in the news articles corpus consisting of '
          '337 words on average.')


fig, ax = plt.subplots(figsize=(10, 6))
metro.year.value_counts().sort_index().plot(ax=ax, style='o-', linewidth=3)
ax.set_xlabel('Year', fontsize=15)
ax.set_ylabel('Number of Articles', fontsize=15)
ax.grid()
plt.show()

fig_label('Trend of Article Count in Inquirer Metro News per Year',
          'There was a decrease in the number of published articles from 2016'
          ' to 2021.')


# Define the list of stopwords
stop_words = (list(set(stopwords.words('english') + list(STOPWORDS)))
              + ['one', 'two', 'three', 'four', 'five', 'six', 'seven',
                 'eight', 'nine', 'let'])

# Instantiate the lemmatizer
lemmatizer = WordNetLemmatizer()

# Drop irrelevant tokens (identified through inspection).
drop_cols = ['around', 'added', 'told', 'still', 'found', 'may', 'de', 'san',
             'even', 'asked', 'along', 'including', 'among', 'take', 'due',
             'ng', 'made', 'took', 'yet', 'went', 'want', 'without', 'saying',
             'na', 'told inquirer', 'sa', 'able', 'state', 'say', 'led',
             'see', 'within', 'put', 'must', 'city', 'said']

# Define the parameters of TfidfVectorizer
tfidf_params = dict(
    ngram_range=(1, 2),
    token_pattern=r'[a-z-]{2,}',
    stop_words=stop_words,
    max_df=0.975,
    min_df=0.025,
    max_features=None
)


# Lemmatize the text
metro.text = metro.text.apply(lambda x: clean_text(x, stop_words, lemmatizer))

# Vectorize the corpus
corpus = vectorize(metro.text, TfidfVectorizer, tfidf_params)

# Get the top tokens in the corpus
tokens = get_token_counts(corpus)


V = corpus[tokens.index].drop(columns=drop_cols)

k_range = 15

try:
    with open('error_list.pkl', 'rb') as f:
        error_list = pickle.load(f)
except:
    error_list = []
    for k in trange(1, k_range+1):
        nmf_params = dict(
            n_components=k,
            solver='mu',
            beta_loss='frobenius',
            max_iter=10_000,
            random_state=6
        )
        nmf = NMF(**nmf_params)
        W = nmf.fit_transform(V)
        H = nmf.components_.T

        error = np.sqrt(np.sum((V.to_numpy() - (W @ H.T))**2))
        error_list.append(error)

        with open('error_list.pkl', 'wb') as f:
            pickle.dump(error_list, f)

fig, ax = plt.subplots(figsize=(10, 6))
ax.plot(range(1, k_range+1), error_list, 'o-')
ax.set_xlabel('Number of Latent Factors', fontsize=15)
ax.set_ylabel('Error', fontsize=15)
ax.hlines(y=error_list[5], xmin=1, xmax=15,
          colors='red', linestyles='--')
ax.vlines(x=6, ymin=min(error_list), ymax=max(error_list),
          colors='red', linestyles='--')
plt.show()

fig_label('Plot of Number of Latent Factors vs The Frobenius Norm Error',
          'A subtle knee can be observed at 3 LFs.')


V = corpus[tokens.index].drop(columns=drop_cols)

n = 6

nmf = NMF(
    n_components=n,
    solver='mu',
    beta_loss='frobenius',
    max_iter=10_000,
    random_state=6
)
W = nmf.fit_transform(V)
H = nmf.components_.T

plot_top_words(nmf, V.columns, nmf.n_components_, 20)

fig_label('Latent Factors in the Corpus',
          'The top weighted words for LF1 to LF6 are police, covid-,'
          ' government, mmda, ltfrb, and fire, respectively.')


topic_dict = {
    0: 'Drug War', 1: 'COVID-19', 2: 'Politics',
    3: 'Traffic Incidents', 4: 'Transportation', 5: 'Fire Incidents'
}
mask = np.array(Image.open("inq.png"))
plot_top_words_wc(nmf, V.columns, nmf.n_components_, 300, topic_dict, mask)

fig_label('Topics in News Articles during Duterte Administration', 
          'Drug War, COVID-19, Politics, Traffic Incidents, Transportation, '
          'and Fire Incidents are the prevailing news topics during the '
          'Duterte administration.')


# Generate trend plot for Results and Discussion

groupings = (pd.concat([metro[['year', 'title']],
                       pd.Series(W.argmax(axis=1), name='topic')], axis=1)
             .groupby(['topic', 'year']).size())

trendline, ax = plt.subplots(figsize=(15, 8))

topics = [
    'Drug War', 'COVID-19', 'Politics',
    'Traffic Incidents', 'Transportation', 'Fire Incidents'
]

articles_per_year = metro.year.value_counts().sort_index()

for i, topic in enumerate(topics):
    (groupings.loc[i] / articles_per_year).plot(ax=ax, style='o-',
                                                label=topic)
#     ax.set_title('Trend of News Topics by Year', fontsize=25)
    ax.set_xlabel('Year', fontsize=20)
    ax.set_ylabel('Proportion', fontsize=20)
    ax.legend()
    ax.grid()
    
trendline.canvas.draw()
plt.close()


years = np.sort(metro.year.unique())

year_dict = {}
for yr in years:
    corpus = vectorize(
        metro[metro.year==yr].text, TfidfVectorizer, tfidf_params
    )
    V = corpus.drop(columns=drop_cols, errors='ignore')
    nmf=NMF(
        n_components=6,
        solver='mu',
        beta_loss='frobenius',
        max_iter=10_000,
        random_state=6
    )
    W=nmf.fit_transform(V),
    H=nmf.components_.T
    year_dict.update({
        yr: dict(corpus=corpus, V=V, nmf=nmf, W=W, H=H)
    })


yr = 2020
plot_top_words(
        year_dict[yr]['nmf'],
        year_dict[yr]['V'].columns,
        year_dict[yr]['nmf'].n_components_,
        20
)

fig_label(f'Latent Factors in the {yr} Subcorpus', 
          'The top weighted words for LF1 to LF6 are coronavirus, police,'
          ' bus, vaccine, hospital, and school, respectively.')


# Topic List
lf_names_dict = {
    2016: {0: 'Drug War', 1: 'Traffic Incidents', 2: 'Government',
           3: 'Fire Incidents', 4: 'PNP Police', 5: 'Quezon City Police'},
    2017: {0: 'Killing Incidents', 1: 'Government', 2: 'Transportation',
           3: 'Traffic Incidents', 4: 'Drug War', 5: 'Rail Transit'},
    2018: {0: 'Police Operations', 1: 'Traffic Incidents',
           2: 'Makati Election', 3: 'Manila Water', 4: 'Transportation',
           5: 'Drug War'},
    2019: {0: 'COVID-19', 1: 'Police Operations', 2: 'Politics',
           3: 'Traffic Incidents', 4: 'Fire Incidents', 5: 'Drug War'},
    2020: {0: 'COVID-19 Cases', 1: 'Drug War', 2: 'Transportation',
           3: 'Vaccination', 4: 'COVID-19 Testing', 5: 'Education'},
    2021: {0: 'COVID-19', 1: 'Drug War', 2: 'Fire Incidents', 3: 'Quarantine',
           4: 'Politics', 5: 'Transportation'}
}
mask = np.array(Image.open("inq.png"))
yr = 2020
plot_top_words_wc(
    year_dict[yr]['nmf'],
    year_dict[yr]['V'].columns,
    year_dict[yr]['nmf'].n_components_,
    300,
    lf_names_dict[yr],
    mask
)
fig_label(f'Topics in News Articles during {yr}', 
          'COVID-19 Cases, Drug War, Transportation, Vaccination,'
          ' COVID-19 Testing, and Education are the prevailing news topics '
          'during the Duterte administration.')


display(trendline)

fig_label('Trend of News Topics by Year',
          'The top topics chronologically from 2016 to 2021 are Drug War, '
          'Politics, COVID-19, and Politics, respectively.')


plot_dist_year(year_dict, lf_names_dict)

fig_label('Top News Topics for Each Year',
          'The highest proportion of Topics for 2016 to 2021 are Government, '
          'Government, Police Operations, Politics, COVID-19 Cases, and '
          'Politics, respectively.')


fig_label('Plot of the Top Occuring Words in each LF by Yearly Subcorpus', '')

for yr in years:
    display(HTML('<center>'+f'<font size="+2"/><b/>{yr}'+'</center>'))
    plot_top_words(
        year_dict[yr]['nmf'],
        year_dict[yr]['V'].columns,
        year_dict[yr]['nmf'].n_components_,
        20,
    )


fig_label('Word clouds of the Top Occuring Words in each LF by Yearly'
          ' Subcorpus', '')

for yr in years:
    display(HTML('<center>'+f'<font size="+2"/><b/>{yr}'+'</center>'))
    plot_top_words_wc(
        year_dict[yr]['nmf'],
        year_dict[yr]['V'].columns,
        year_dict[yr]['nmf'].n_components_,
        300,
        lf_names_dict[yr],
        mask,
    )

Feature	Datatype	Description
`title`	object	Title of the news article
`link`	object	Web link of the news article
`date`	object	Publish date (YY-MM-dd)
`year`	int	Publish year
`text`	object	Content of the news article

Executive Summary¶

Introduction¶

Background of the Study¶

Problem Statement¶

Business Value¶

Scope and Limitations¶

Methodology¶

Overview¶

Data Acquisition¶

Data Preprocessing¶

Exploratory Data Analysis¶

Analysis of Corpus¶

Vectorization¶

Non-negative Matrix Factorization¶

Theme Analysis¶

Analysis of Yearly Subcorpora¶

Vectorization¶

Non-negative Matrix Factorization¶

Theme Analysis¶

Results and Discussion¶

Analysis of Corpus¶

Analysis of Yearly Subcorpora¶

Conclusion¶

Recommendations¶

References¶

Appendix¶